{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorflow的核心概念自动微分Autodiff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. 简单的求导"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对于公式：y = $x^2$ + 4x，求y对x的导数，即y=2x+4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(10.0, shape=(), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-03-24 19:31:20.542180: I tensorflow/core/platform/cpu_feature_guard.cc:151] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
     ]
    }
   ],
   "source": [
    "x = tf.Variable(3.)\n",
    "\n",
    "with tf.GradientTape() as t:\n",
    "    y = x*x + 4*x\n",
    "\n",
    "# 导数应该是：2*x + 4 = 2*3 + 4 = 9\n",
    "dy_dx = t.gradient(y, x)\n",
    "print(dy_dx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. 在模型中的使用"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 构造数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3, 4) (3,)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "features = np.array([\n",
    "    [1,2,3,4],\n",
    "    [5,6,7,8],\n",
    "    [9,10,11,12]\n",
    "])\n",
    "\n",
    "labels = np.array([1, 0, 1])\n",
    "\n",
    "print(features.shape, labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 搭建一个model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential(\n",
    "    [\n",
    "        tf.keras.layers.Dense(3, input_shape=(4,)),\n",
    "        tf.keras.layers.Dense(1, activation=\"sigmoid\")\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense/kernel:0' shape=(4, 3) dtype=float32, numpy=\n",
       " array([[ 0.32193863, -0.3694399 , -0.12214905],\n",
       "        [ 0.37250054, -0.76891845,  0.5325408 ],\n",
       "        [-0.5349232 ,  0.49259067, -0.05792421],\n",
       "        [-0.4529848 ,  0.7604511 , -0.29454595]], dtype=float32)>,\n",
       " <tf.Variable 'dense/bias:0' shape=(3,) dtype=float32, numpy=array([0., 0., 0.], dtype=float32)>,\n",
       " <tf.Variable 'dense_1/kernel:0' shape=(3, 1) dtype=float32, numpy=\n",
       " array([[-1.1459512 ],\n",
       "        [-0.55327976],\n",
       "        [ 0.5144625 ]], dtype=float32)>,\n",
       " <tf.Variable 'dense_1/bias:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 待更新的参数，已经进行了初始化\n",
    "model.trainable_weights"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 自动计算微分更新参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# loss function \n",
    "loss_func=tf.keras.losses.BinaryCrossentropy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logits:\n",
      " tf.Tensor(\n",
      "[[0.7382632]\n",
      " [0.9044391]\n",
      " [0.969473 ]], shape=(3, 1), dtype=float32)\n",
      "\n",
      "loss_value:\n",
      " tf.Tensor(0.8941495, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "with tf.GradientTape(persistent=True) as tape:\n",
    "    # 对于features，经过model计算，输出logits\n",
    "    logits = model(features)\n",
    "    print(\"logits:\\n\", logits)\n",
    "    \n",
    "    loss_value = loss_func(labels.reshape(3,1), logits)\n",
    "    print()\n",
    "    print(\"loss_value:\\n\", loss_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Tensor: shape=(4, 3), dtype=float32, numpy=\n",
       " array([[-1.5224783 , -0.73507184,  0.68350035],\n",
       "        [-1.7563194 , -0.8479732 ,  0.78848064],\n",
       "        [-1.9901602 , -0.9608746 ,  0.89346105],\n",
       "        [-2.2240012 , -1.0737759 ,  0.9984414 ]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(3,), dtype=float32, numpy=array([-0.23384097, -0.11290138,  0.10498038], dtype=float32)>,\n",
       " <tf.Tensor: shape=(3, 1), dtype=float32, numpy=\n",
       " array([[-0.8094997 ],\n",
       "        [ 0.6620247 ],\n",
       "        [-0.01833116]], dtype=float32)>,\n",
       " <tf.Tensor: shape=(1,), dtype=float32, numpy=array([0.20405838], dtype=float32)>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 计算梯度，\n",
    "gradients = tape.gradient(loss_value, model.trainable_weights)\n",
    "gradients"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 使用计算的梯度更新模型的参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Variable 'UnreadVariable' shape=() dtype=int64, numpy=1>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 使用优化器，给变量model.trainable_weights应用梯度\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "optimizer.apply_gradients(zip(gradients, model.trainable_weights))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<tf.Variable 'dense/kernel:0' shape=(4, 3) dtype=float32, numpy=\n",
       " array([[ 0.32293862, -0.3684399 , -0.12314904],\n",
       "        [ 0.37350053, -0.76791847,  0.5315408 ],\n",
       "        [-0.5339232 ,  0.49359065, -0.05892421],\n",
       "        [-0.45198482,  0.76145107, -0.29554594]], dtype=float32)>,\n",
       " <tf.Variable 'dense/bias:0' shape=(3,) dtype=float32, numpy=array([ 0.00099999,  0.00099997, -0.00099997], dtype=float32)>,\n",
       " <tf.Variable 'dense_1/kernel:0' shape=(3, 1) dtype=float32, numpy=\n",
       " array([[-1.1449511 ],\n",
       "        [-0.55427974],\n",
       "        [ 0.5154623 ]], dtype=float32)>,\n",
       " <tf.Variable 'dense_1/bias:0' shape=(1,) dtype=float32, numpy=array([-0.00099998], dtype=float32)>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 模型参数，我们会看到进行了更新\n",
    "model.trainable_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
